const cheerio = require('cheerio')

const http = require('http')

const fs = require('fs')

const request = require('request')

let i = 0

let abs_url = "http://www.ss.pku.edu.cn"

let url = `${abs_url}/index.php/newscenter/news/2391`

function fetchPage(x) {
    startRequest(x); 
}

function startRequest(x) {
	http.get(x,(res)=> {
		let html = ""
		let titles = []
		let $ = ""
		res.setEncoding ("UTF8")
		res.on('data',(chunk) => {
			html += chunk
		})
		res.on('end',() => {
			i = i+1
			$ = cheerio.load(html)
			let time = $('.icon-calendar').parent().text().trim()
			let title = $('.article-title a').text().trim()
			let link = abs_url + $("div.article-title a").attr('href')
			let author = $("[title=供稿]").text().trim()
			let news_item = { time,title,link,author,i }
			let nextPageLink = abs_url + $("li.next a").attr("href")
			nextPageLink = nextPageLink.split('-')[0]
			saveContent($,title)
			saveImg($)
			// console.log(nextPageLink)
			if(i<20) {
				setTimeout(()=>{
					fetchPage(nextPageLink)
				},6000)
			}
		})
	})
}

function saveContent($,news_title) {
	$('.article-content p').each((index, el) => {
		let content = $(el).text()
		let subCnt = content.substring(0,2).trim()
		if(subCnt=='') {
			let path = `./data/${news_title}.txt`
			content = content += '\n'
			fs.appendFile(path,content,'utf-8',(err) => {
				if(err) {
					console.log(err)
				}
			})
		}

	});
}

function saveImg($) {
	$('.article-content img').each((index,el) => {
		let img_title = subName( $(el).parent().next().text().trim() )
		let img_url = $(el).attr('src')
		let img_src = `${abs_url}${img_url}`
		let tailName = img_url.replace(/.+\./,"")
		let img_file_name = subName( img_title+'.'+tailName )
		let writeStm = fs.createWriteStream(`./image/${img_title}--${img_file_name}`)
		request(img_src).pipe(writeStm)
		// console.log(writeStm)
		console.log(`tailName:${tailName}`)
	})
	function subName(str) {
		str = str ? str.substring(0,5) :'Null'
		return str
	}
}


fetchPage(url); 